from IPython.display import display, HTML
HTML(
    """
    <script
        src='https://cdnjs.cloudflare.com/ajax/libs/jquery/2.0.3/jquery.min.js'>
    </script>
    <script>
        code_show=true;
        function code_toggle() {
        if (code_show){
        $('div.jp-CodeCell > div.jp-Cell-inputWrapper').hide();
        } else {
        $('div.jp-CodeCell > div.jp-Cell-inputWrapper').show();
        }
        code_show = !code_show
        }
        $( document ).ready(code_toggle);
    </script>
    <form action='javascript:code_toggle()'>
        <input type="submit" value='Click here to toggle on/off the raw code.'>
    </form>
    """
)

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

from surprise import Dataset, Reader
from surprise.model_selection import train_test_split
from surprise import KNNBasic
from surprise import accuracy

import joblib
import random
import warnings

from sklearn.feature_extraction.text import TfidfVectorizer
import nltk
from nltk.stem import WordNetLemmatizer
from nltk.corpus import stopwords
import re

from scipy.spatial.distance import cosine

# Suppress specific RuntimeWarning
warnings.filterwarnings("ignore", category=RuntimeWarning)

print('Table 1. Anime dataset column names and their associated brief description.')

Table 1. Anime dataset column names and their associated brief description.

print('Table 2. User-Anime Interaction dataset columns and their brief description.')

Table 2. User-Anime Interaction dataset columns and their brief description.

df_anime = pd.read_csv("anime-dataset-2023.csv")
df_user = pd.read_csv("users-score-2023.csv")

print('Table 3. Number of null values in anime dataset.')

Table 3. Number of null values in anime dataset.

pd.DataFrame(df_anime.isna().sum(), columns=['Number of null values'])

print('Table 4. Number of null values in user-anime interaction dataset.')

Table 4. Number of null values in user-anime interaction dataset.

pd.DataFrame(df_user.isna().sum(), columns=['Number of null values'])

print(f'Number of duplicates in anime dataset: {df_anime.drop("Genres", axis=1).duplicated().sum()}')
print(f'Number of duplicates in user-anime interaction dataset: {df_user.duplicated().sum()}')

Number of duplicates in anime dataset: 0
Number of duplicates in user-anime interaction dataset: 0

plt.figure(figsize=(10, 6))

df_user.rating.plot(kind='hist', color='#F05A39')
plt.xlabel('Rating')
plt.xticks(rotation=45)
plt.tight_layout()
plt.show()

print(f'  3. Mean Global Rating: {df_user.rating.mean(): .1f}')

  3. Mean Global Rating:  7.6

def plot_top_anime(df, n=10):
    """
    Plot the top N anime by average rating and number of votes.
    """
    anime_stats = df.groupby('Anime Title').agg({'rating': ['mean', 'count']})
    anime_stats.columns = ['Average Rating', 'Number of Votes']
    top_anime = anime_stats.sort_values(
        by=['Number of Votes', 'Average Rating'], ascending=False).head(n)
    top_anime_sorted = top_anime.sort_values(
        by='Average Rating', ascending=True)
    plt.figure(figsize=(10, 6))
    top_anime_sorted['Average Rating'].plot(kind='barh', color='#F05A39')
    plt.xlabel('Rating')
    plt.tight_layout()
    plt.show()

    return top_anime_sorted

top_anime_sorted_idx = plot_top_anime(df_user, n=10)

df_anime_2 = df_anime.set_index('Name')
top_anime_10_df = df_anime_2.loc[top_anime_sorted_idx.index.tolist()[::-1], [
    'Genres', 'Studios', 'Producers']]
print('Table 5. Top 10 Anime with their genres, studios, and producers.')
top_anime_10_df

Table 5. Top 10 Anime with their genres, studios, and producers.

print('Table 6. Snippet of anime dataset with anime with no description.')
df_anime.query("Synopsis=='No description available for this anime.'").head(
    3).drop('Other name', axis=1)

Table 6. Snippet of anime dataset with anime with no description.

df_anime_filtered = df_anime[df_anime['Synopsis']
                             != 'No description available for this anime.']

def create_util_matrix(df_anime, df_user, sample_size=10000, random_state=42):
    anime_idx = df_anime['anime_id'].tolist()
    df_user_content = df_user[df_user['anime_id'].isin(anime_idx)]
    user_idx_content = df_user_content.groupby('user_id').size()
    top_user_idx = user_idx_content.sample(
        sample_size, random_state=random_state).index
    df_rand_user = df_user_content[df_user_content['user_id'].isin(
        top_user_idx)]
    pivot_user = df_rand_user.pivot_table(
        values="rating", index="user_id", columns="anime_id")
    return pivot_user, df_rand_user, user_idx_content

df_utility, df_rand_user, user_idx_content = create_util_matrix(
    df_anime_filtered, df_user, sample_size=10000, random_state=42)
print('Table 7. A snippet of User-Anime Matrix')
df_utility.iloc[:5, :20]

Table 7. A snippet of User-Anime Matrix

anime_idx = df_utility.columns.to_list()
filtered_anime_df1 = df_anime_filtered[df_anime_filtered['anime_id'].isin(anime_idx)]

# Ensure that NLTK resources are downloaded (do this once)
# nltk.download('wordnet')
# nltk.download('stopwords')

# Lemmatization function
def lemmatize_text(text):
    lemmatizer = WordNetLemmatizer()
    word_list = nltk.word_tokenize(text)
    return ' '.join([lemmatizer.lemmatize(w) for w in word_list])


# Suppress SettingWithCopyWarning
pd.options.mode.chained_assignment = None  # default='warn'


def create_tfidf(df_anime, df_utility):
    filtered_anime_df1['Synopsis'] = filtered_anime_df1['Synopsis'].map(
        lambda text: re.sub('[^a-zA-Z]', ' ', text).lower())
    filtered_anime_df1['Synopsis'] = filtered_anime_df1['Synopsis'].apply(
        lemmatize_text)
    vectorizer = TfidfVectorizer(stop_words=stopwords.words('english'),
                                 min_df=0.01,
                                 max_df=0.95)
    tfidf_matrix = vectorizer.fit_transform(filtered_anime_df1['Synopsis'])
    return pd.DataFrame(tfidf_matrix.toarray(),
                        columns=vectorizer.get_feature_names_out(),
                        index=df_utility.columns)

print('Table 8. Snippet of TF-IDF matrix of anime synopses.')
tfidf_df = create_tfidf(df_anime, df_utility)
tfidf_df.iloc[:5, :20]

Table 8. Snippet of TF-IDF matrix of anime synopses.

def recommend_collab(df_rand_user, user_based=True):
    reader = Reader(rating_scale=(1, 10))
    data = Dataset.load_from_df(
        pd.DataFrame(df_rand_user, columns=['user_id', 'anime_id', 'rating']),
        reader)
    trainset, testset = train_test_split(data, test_size=0.25, random_state=26)
    sim_options = {'name': 'pearson', 'user_based': user_based}
    algo = KNNBasic(sim_options=sim_options, k=50, min_k=5, verbose=False)
    algo.fit(trainset)
    predictions = algo.test(testset)
    return predictions, algo

predictions_user, algo_user = recommend_collab(df_rand_user, user_based=True)
predictions_item, algo_item = recommend_collab(df_rand_user, user_based=False)

# Save the trained model to a file
joblib.dump(algo_user, 'collaborative_filtering_model.pkl')
loaded_model = joblib.load('collaborative_filtering_model.pkl')

# Save the trained model to a file
joblib.dump(algo_item, 'collaborative_filtering_model_item.pkl')
loaded_model_item = joblib.load('collaborative_filtering_model_item.pkl')

def compute_user_profile_agg_numeric(df_utility, df_item_profiles, user):
    user_ratings = df_utility.loc[user]
    mean_rating = user_ratings.mean()
    centered_ratings = user_ratings - mean_rating
    weighted_profiles = df_item_profiles.mul(centered_ratings, axis=0)
    user_profile = weighted_profiles.mean()
    return user_profile

def recommend_agg_numeric(df_utility, df_item_profiles, user_profile, user):
    nan_idx = np.isnan(df_utility.loc[user])
    items = df_item_profiles.loc[nan_idx]
    ratings = sorted(
        [
            (i, cosine(item, user_profile))
            for i, item in items.iterrows()
            if cosine(item, user_profile) > 0
        ],
        key=lambda x: (x[1], x[0]),
    )
    return [i for i, _ in ratings]

def recommended_anime_content(list_id, df_anime):
    df_anime_reco = pd.DataFrame(columns=df_anime.columns)
    
    for id in list_id:
        row_reco = df_anime[df_anime['anime_id']==id]
        df_anime_reco = pd.concat([df_anime_reco, row_reco], ignore_index=True)
        
    return df_anime_reco

user = 21142
user_profile = compute_user_profile_agg_numeric(df_utility, tfidf_df, user)

def get_reco_cf(df_top_user, user, algo, df_anime):
    # this is original anime reco, without filters, so that no bias or subjective reasoning integrated
    user_items = df_top_user[df_top_user['user_id']
                             == user]['anime_id'].tolist()
    all_items = df_top_user['anime_id'].unique()
    not_rated_items = [item for item in all_items if item not in user_items]
    recommendations = [algo.predict(user, item) for item in not_rated_items]
    recommendations.sort(key=lambda x: x.est, reverse=True)
    recommended_items = [pred.iid for pred in recommendations]
    df_anime_reco = pd.DataFrame(columns=df_anime.columns)

    for id in recommended_items:
        row_reco = df_anime[df_anime['anime_id'] == id]
        df_anime_reco = pd.concat([df_anime_reco, row_reco], ignore_index=True)

    return df_anime_reco

def filter_recos(df, n=10):
    to_exclude_genres = [
        'Boys Love',
        'Ecchi',
        'Erotica',
        'Girls Love',
        'UNKNOWN'
    ]
    to_exclude_ratings = [
        # 'R - 17+ (violence & profanity)',
        'R+ - Mild Nudity',
        'Rx - Hentai',
        'UNKNOWN']

    to_exclude_type = [
        'Special',
        'Music',
        'UNKNOWN'
    ]
    def should_include(genres_list):
        return not any(genre in to_exclude_genres for genre in genres_list)

    to_exclude_rank = ['0.0', 'UNKNOWN']
    df['Genres'] = df['Genres'].str.split(', ')
    df = df[df['Genres'].apply(should_include)]
    df = df[~df['Rating'].isin(to_exclude_ratings)]
    df = df[~df['Type'].isin(to_exclude_type)]
    df = df[df["Synopsis"] != "No description available for this anime."]
    df = df[~df['Rank'].isin(to_exclude_rank)]
    df = df[df['Popularity']!=0]
    df['Rank'] =  pd.to_numeric(df['Rank'], downcast='integer')
    df = df[~df['Rank']<100]
    return df.head(n)

df_anime_reco_user = get_reco_cf(df_rand_user, user, loaded_model, df_anime)
user_cf_recos = filter_recos(df_anime_reco_user).Name.tolist()

df_anime_reco_item = get_reco_cf(df_rand_user, user, loaded_model_item, df_anime)
item_cf_recos = filter_recos(df_anime_reco_item).Name.tolist()

list_anime = recommend_agg_numeric(df_utility, tfidf_df, user_profile, user)
cb_recos = recommended_anime_content(list_anime, df_anime).Name.tolist()[:10]

anime_rated = df_user.query("user_id==21142")[
    'Anime Title'].tolist() + (['-'] * 3)

print('Table 9. Top 10 anime recommendations across all methods.')
pd.DataFrame({
    'Anime Rated': anime_rated,
    'User-based CF': user_cf_recos,
    'Item-based CF': item_cf_recos,
    'Content-based': cb_recos
})

Table 9. Top 10 anime recommendations across all methods.

def get_mae_baseline(df_utility):    
    user_count = df_utility.count(axis=1)
    user_at_least_10 = user_count[user_count>=10]
    mae_list = []
    for idx in user_at_least_10.index:
        random.seed(42)
        user_rated = df_utility.loc[idx].dropna()
        random_10_idx = random.sample(user_rated.index.tolist(), 10)
        remove_5_idx = random.sample(random_10_idx, 5)
        ground_truth = user_rated.loc[remove_5_idx]
        naive_ratings = user_rated.loc[random_10_idx]
        naive_ratings.loc[remove_5_idx] = naive_ratings.drop(remove_5_idx).median()
        mae = abs(ground_truth - naive_ratings.loc[remove_5_idx]).mean()
        mae_list.append(mae)
    return pd.Series(mae_list).mean()

mae_baseline = round(get_mae_baseline(df_utility), 2)

mae_user = round(accuracy.mae(predictions_user, verbose=False), 2)
mae_item = round(accuracy.mae(predictions_item, verbose=False), 2)

print('Table 9. Summary of MAE for Collaborative Filtering and Naive')
display(pd.DataFrame({
    'Model': ['User-based CF', 'Item-based CF', 'Naive'],
    'MAE': [mae_user, mae_item, mae_baseline]
}))

Table 9. Summary of MAE for Collaborative Filtering and Naive

def ndcg_custom(df, rand_user):
    dcg = 0
    idcg = 0 
    for i, data in df.iterrows():
        idcg += (2**data[rand_user]-1) / (np.log2(i+2))
        dcg += (2**data['score_2']-1) / (np.log2(i+2))
    return dcg/idcg

def get_ndcg_cb(df_utility, tfidf_df, rand_user, seed=42):
    id_with_ratings = df_utility.loc[rand_user].dropna().sort_values(
        ascending=False).index.tolist()
    length_unrated = len(df_utility.loc[rand_user]) - len(
        df_utility.loc[rand_user].dropna())
    random.seed(seed) 
    random_choices = random.sample(id_with_ratings, 15)
    random_ratings = df_utility.loc[rand_user, random_choices].sort_values(
        ascending=False).reset_index()
    ideal = random_ratings.sort_values(by=[rand_user, 'anime_id'],
                                       ascending=[False, True])
    df_utility_2 = df_utility.copy()
    df_utility_2.loc[rand_user, random_choices] = np.nan
    user_profile_2 = compute_user_profile_agg_numeric(df_utility_2, tfidf_df,
                                                      rand_user)
    list_anime_2 = recommend_agg_numeric(df_utility_2, tfidf_df, user_profile_2,
                                         rand_user, length_unrated)
    indices = [
        list_anime_2.index(elem)
        for elem in random_choices
        if elem in list_anime_2
    ]
    zipped_list = list(zip(indices, random_choices))
    rec_rankings = sorted(zipped_list, key=lambda x: x[0])
    ideal['anime_id_2'] = [item[1] for item in rec_rankings]
    # Mapping scores from anime_id to anime_id_2 to create the score_2 column
    score_mapping = ideal.set_index('anime_id')[rand_user].to_dict()
    ideal['score_2'] = ideal['anime_id_2'].map(score_mapping)
    return ideal

def compute_mean_ndcg(user_idx_content, df_utility, tfidf_df):
    user_id = user_idx_content.sort_values(ascending=False)[:1600].index
    users_util = df_utility.index
    fifty_users = [value for value in user_id if value in users_util]
    ndcg_scores = []
    for user in fifty_users:
        df_id_scores = get_ndcg_cb(df_utility, tfidf_df, rand_user=user)
        ndcg = ndcg_custom(df_id_scores, rand_user=user)
        ndcg_scores.append(ndcg)
    return pd.Series(ndcg_scores).mean()

mean_ndcg_cb = compute_mean_ndcg(user_idx_content, df_utility, tfidf_df)

Column	Description
anime_id	Unique ID for each anime.
Name	The name of the anime in its original language.
English name	The English name of the anime.
Other name	Native name or title of the anime (e.g., Japanese, Chinese, Korean).
Score	The score or rating given to the anime. (overall popularity score)
Genres	The genres of the anime, separated by commas.
Synopsis	A brief description or summary of the anime's plot.
Type	The type of the anime (e.g., TV series, movie, OVA, etc.).
Episodes	The number of episodes in the anime.
Aired	The dates when the anime was aired.
Premiered	The season and year when the anime premiered.
Status	The status of the anime (e.g., Finished Airing, Currently Airing, etc.).
Producers	The production companies or producers of the anime.
Licensors	The licensors of the anime (e.g., streaming platforms).
Studios	The animation studios that worked on the anime.
Source	The source material of the anime (e.g., manga, light novel, original).
Duration	The duration of each episode.
Rating	The age rating of the anime.
Rank	The rank of the anime based on popularity or other criteria.
Popularity	The popularity rank of the anime.
Favorites	The number of times the anime was marked as a favorite by users.
Scored By	The number of users who scored the anime.
Members	The number of members who have added the anime to their list on the platform.
Image URL	The URL of the anime's image or poster.

Column	Description
user_id	Unique ID for each user.
Username	The username of the user.
anime_id	Unique ID for each anime.
Anime Title	The title of the anime.
rating	The rating given by the user to the anime.

	Number of null values
anime_id	0
Name	0
English name	0
Other name	0
Score	0
Genres	0
Synopsis	0
Type	0
Episodes	0
Aired	0
Premiered	0
Status	0
Producers	0
Licensors	0
Studios	0
Source	0
Duration	0
Rating	0
Rank	0
Popularity	0
Favorites	0
Scored By	0
Members	0
Image URL	0

	Genres	Studios	Producers
Name
Fullmetal Alchemist: Brotherhood	Action, Adventure, Drama, Fantasy	Bones	Aniplex, Square Enix, Mainichi Broadcasting Sy...
Code Geass: Hangyaku no Lelouch R2	Action, Award Winning, Drama, Sci-Fi	Sunrise	Bandai Visual, Mainichi Broadcasting System, B...
Sen to Chihiro no Kamikakushi	Adventure, Award Winning, Supernatural	Studio Ghibli	Toho
Code Geass: Hangyaku no Lelouch	Action, Award Winning, Drama, Sci-Fi	Sunrise	Bandai Visual, Mainichi Broadcasting System, B...
Death Note	Supernatural, Suspense	Madhouse	VAP, Konami, Ashi Productions, Nippon Televisi...
Fullmetal Alchemist	Action, Adventure, Award Winning, Drama, Fantasy	Bones	Aniplex, Dentsu, Square Enix, Mainichi Broadca...
Angel Beats!	Drama, Supernatural	P.A. Works	Aniplex, Dentsu, Mainichi Broadcasting System,...
Elfen Lied	Action, Drama, Horror, Romance, Supernatural	Arms	VAP, Genco, Studio Guts
Bleach	Action, Adventure, Fantasy	Pierrot	TV Tokyo, Aniplex, Dentsu, TV Tokyo Music, Stu...
Naruto	Action, Adventure, Fantasy	Pierrot	TV Tokyo, Aniplex, Shueisha

	anime_id	Name	English name	Score	Genres	Synopsis	Type	Episodes	Aired	Premiered	...	Studios	Source	Duration	Rating	Rank	Popularity	Favorites	Scored By	Members	Image URL
2412	2626	The☆Doraemons: Mushimushi Pyonpyon Daisakusen!	UNKNOWN	6.35	Fantasy	No description available for this anime.	Movie	1.0	Mar 7, 1998	UNKNOWN	...	Shin-Ei Animation	Unknown	16 min	PG - Children	7118.0	11312	0	652.0	1583	https://cdn.myanimelist.net/images/anime/11/31...
2413	2628	The☆Doraemons: Strange, Sweets, Strange?	UNKNOWN	6.11	Fantasy	No description available for this anime.	Movie	1.0	Mar 6, 1999	UNKNOWN	...	Shin-Ei Animation	Unknown	16 min	PG - Children	8295.0	11634	0	531.0	1405	https://cdn.myanimelist.net/images/anime/1400/...
2414	2629	Doraemon and Itchy the Stray	UNKNOWN	6.48	Comedy, Sci-Fi	No description available for this anime.	Special	1.0	Jan 2, 1987	UNKNOWN	...	UNKNOWN	Unknown	22 min	PG - Children	6467.0	10959	1	775.0	1814	https://cdn.myanimelist.net/images/anime/12/57...

Overview¶

Methodology Overview¶

Detailed¶

Exploratory Data Analysis¶

Data Pre-processing¶

Recommender System Development¶

Performance Evaluation¶

anime_id	1	5	6	7	8	15	16	17	18	19	20	21	22	23	24	25	26	27	28	29
user_id
71	NaN	NaN	8.0	NaN	NaN	NaN	NaN	NaN	8.0	9.0	7.0	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN
376	7.0	8.0	9.0	NaN	NaN	NaN	8.0	NaN	NaN	NaN	10.0	10.0	NaN	NaN	8.0	NaN	NaN	7.0	NaN	NaN
380	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	9.0	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN
429	NaN	NaN	NaN	NaN	7.0	NaN	NaN	NaN	NaN	NaN	7.0	10.0	NaN	NaN	NaN	NaN	NaN	NaN	5.0	NaN
475	10.0	NaN	NaN	NaN	NaN	8.0	NaN	NaN	NaN	7.0	8.0	10.0	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN

	abandoned	ability	able	academy	accident	accidentally	across	act	action	activity	actually	adult	adventure	age	ago	ai	aim	aired	alien	ally
anime_id
1	0.0	0.0	0.000000	0.0	0.0	0.0	0.141130	0.0	0.000000	0.0	0.0	0.0	0.000000	0.0	0.0	0.0	0.0	0.0	0.0	0.0
5	0.0	0.0	0.000000	0.0	0.0	0.0	0.166721	0.0	0.172253	0.0	0.0	0.0	0.000000	0.0	0.0	0.0	0.0	0.0	0.0	0.0
6	0.0	0.0	0.000000	0.0	0.0	0.0	0.000000	0.0	0.000000	0.0	0.0	0.0	0.000000	0.0	0.0	0.0	0.0	0.0	0.0	0.0
7	0.0	0.0	0.000000	0.0	0.0	0.0	0.000000	0.0	0.000000	0.0	0.0	0.0	0.000000	0.0	0.0	0.0	0.0	0.0	0.0	0.0
8	0.0	0.0	0.139239	0.0	0.0	0.0	0.000000	0.0	0.000000	0.0	0.0	0.0	0.141551	0.0	0.0	0.0	0.0	0.0	0.0	0.0

	Anime Rated	User-based CF	Item-based CF	Content-based
0	Gintama	Gintama'	Jungle wa Itsumo Hare nochi Guu Final	DNA²
1	Fullmetal Alchemist	Fullmetal Alchemist: Brotherhood	Tensei shitara Ken deshita	Geneshaft
2	Howl no Ugoku Shiro	Code Geass: Hangyaku no Lelouch R2	Nagi no Asu kara	SoniAni: Super Sonico The Animation
3	Kuragehime	Shouwa Genroku Rakugo Shinjuu: Sukeroku Futata...	Kyoukai no Rinne 3rd Season	Dance with Devils: Fortuna
4	Mawaru Penguindrum	Mushishi Zoku Shou: Suzu no Shizuku	Mayonaka no Occult Koumuin	Aniyome
5	Bleach	Gintama': Enchousen	Zombieland Saga Revenge	B'T X
6	Kuroshitsuji	Ousama Ranking	Kakuriyo no Yadomeshi	Majo Minarai wo Sagashite
7	-	Gintama Movie 2: Kanketsu-hen - Yorozuya yo Ei...	.hack//Liminality	Megami-ryou no Ryoubo-kun.
8	-	Gintama°	.hack//G.U. Returner	Diabolik Lovers
9	-	Ranma ½ OVA	IDOLiSH7	Oniichan wa Oshimai!

	Model	MAE
0	User-based CF	1.17
1	Item-based CF	1.14
2	Naive	1.20